{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "-8DWs66YCcg4"
   },
   "source": [
    "this is based on https://github.com/dimun/pate_torch/blob/master/PATE.ipynb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "p-JylkvK_Q5z"
   },
   "source": [
    "# Private Aggregation of Teacher Ensembles (PATE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "id": "VEuCLJfuevUj",
    "outputId": "bf05bed5-3c22-4020-cde0-fa53ce590072"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch==1.6.0+cu101\n",
      "torchsummary==1.5.1\n",
      "torchtext==0.3.1\n",
      "torchvision==0.7.0+cu101\n"
     ]
    }
   ],
   "source": [
    "!pip freeze | grep torch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "gLn9eKfe_Q51"
   },
   "source": [
    "## Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "cyjvipBT_Q52"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "import numpy as np\n",
    "from torchvision import datasets\n",
    "import torchvision.transforms as transforms\n",
    "from torch.utils.data import Subset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "1nI-_HBC_Q55"
   },
   "source": [
    "## Load the [Data](http://pytorch.org/docs/stable/torchvision/datasets.html)\n",
    "\n",
    "Downloading may take a few moments, and you should see your progress as the data is loading. You may also choose to change the `batch_size` if you want to load more data at a time."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 369,
     "referenced_widgets": [
      "88c34b495ef84d18868c89d86ba74781",
      "30d23308b628438c9424a68bf73f4468",
      "b22a21cee9b74cf9bf88335f39f5eafc",
      "1b60c8ec78c745b3876a79c53684cf91",
      "bd429a24c41f4f0aa03da63558677bce",
      "3f96f5ab3d6d47bcb7948e36233e4983",
      "0b4fe1a43b2b462c8a83c76764ba9e18",
      "3c6e3eec49b54b369faa256c4440ffdf",
      "e12027df7fdf4427976c5f89d35c78cf",
      "01c73e5d7b3e4c7bae717f0ee51e1058",
      "e3c0820d97734cf9a987f0ddf54fabbc",
      "3d2ff1e00e844fc08f54f19cd2dff484",
      "9eb486ae196945a7b0213f36d8dc3cdf",
      "5f1007296f3c4071ad9c1ee99643f596",
      "b2513595352f433f9a8b7a8cbb30f07b",
      "5a74a153098947f191dfa69ff9b1c2e2",
      "aaf968ebb58d4b88810087c6222c13ba",
      "1a9ee64ef77e4160bc0e59c01099be18",
      "f2f58cbdbe464dd6b4240425ed69a567",
      "2a27418390754d9397bd06b49363809a",
      "596367d5d9a14f6c9a1fd9139dfaa442",
      "4f8b5f7987c84a2a9ddb01f62876a024",
      "a73fa9db86344b59afe151470d01326f",
      "5ed4cdf05f514f6195381c0c6a83df89",
      "94fc494383184117aa3748673053bbe7",
      "20ee51f905aa4576bd7b6df533cb430a",
      "d79e8a56a09545a0b19cb50d1486d81c",
      "0fadbe1391d741f5982f6ff9a88dfb37",
      "74e87bd28def4a0187379d04fb4b83ed",
      "2fc36c2d96ec4ea1b42ce74a40072db1",
      "9d36447c5a564e4cb156efdc209fe012",
      "7bac61dd31f64b56bfa1d2d76ef2e7e9"
     ]
    },
    "colab_type": "code",
    "id": "xDTtX0cf_Q56",
    "outputId": "bb9949c0-d348-4c59-9ec8-0f69b53f70c5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz to data/MNIST/raw/train-images-idx3-ubyte.gz\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "88c34b495ef84d18868c89d86ba74781",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting data/MNIST/raw/train-images-idx3-ubyte.gz to data/MNIST/raw\n",
      "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz to data/MNIST/raw/train-labels-idx1-ubyte.gz\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e12027df7fdf4427976c5f89d35c78cf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting data/MNIST/raw/train-labels-idx1-ubyte.gz to data/MNIST/raw\n",
      "Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz to data/MNIST/raw/t10k-images-idx3-ubyte.gz\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aaf968ebb58d4b88810087c6222c13ba",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting data/MNIST/raw/t10k-images-idx3-ubyte.gz to data/MNIST/raw\n",
      "Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz to data/MNIST/raw/t10k-labels-idx1-ubyte.gz\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "94fc494383184117aa3748673053bbe7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting data/MNIST/raw/t10k-labels-idx1-ubyte.gz to data/MNIST/raw\n",
      "Processing...\n",
      "Done!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/torchvision/datasets/mnist.py:469: UserWarning: The given NumPy array is not writeable, and PyTorch does not support non-writeable tensors. This means you can write to the underlying (supposedly non-writeable) NumPy array using the tensor. You may want to copy the array to protect its data or make it writeable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at  /pytorch/torch/csrc/utils/tensor_numpy.cpp:141.)\n",
      "  return torch.from_numpy(parsed.astype(m[2], copy=False)).view(*s)\n"
     ]
    }
   ],
   "source": [
    "# number of subprocesses to use for data loading\n",
    "num_workers = 0\n",
    "# how many samples per batch to load\n",
    "batch_size = 32\n",
    "\n",
    "# convert data to torch.FloatTensor\n",
    "transform = transforms.Compose(\n",
    "    [transforms.ToTensor(),\n",
    "     transforms.Normalize((0.1307,), (0.3081,))]\n",
    ")\n",
    "\n",
    "# choose the training and test datasets\n",
    "train_data = datasets.MNIST(\n",
    "    root='data',\n",
    "    train=True,\n",
    "    download=True,\n",
    "    transform=transform\n",
    ")\n",
    "test_data = datasets.MNIST(\n",
    "    root='data', train=False,\n",
    "    download=True, transform=transform\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "vcgFJx4l_Q59"
   },
   "source": [
    "Function for returning dataloaders for a specified number of teachers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 68
    },
    "colab_type": "code",
    "id": "dUYhi5o1_Q59",
    "outputId": "48bb2c52-29e1-4cf6-9150-50fa8216a410"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# number of teachers to essemble\n",
    "num_teachers = 100\n",
    "\n",
    "def get_data_loaders(train_data, num_teachers=10):\n",
    "    \"\"\"Simple partitioning algorithm that returns the right portion of the data\n",
    "    needed by a given teacher out of a certain number of teachers.\n",
    "\n",
    "    Each teacher model will get a disjoint subset of the training data.\n",
    "    \"\"\"\n",
    "    teacher_loaders = []\n",
    "    data_size = len(train_data) // num_teachers\n",
    "\n",
    "    for i in range(num_teachers):\n",
    "        indices = list(range(i * data_size, (i+1) * data_size))\n",
    "        subset_data = Subset(train_data, indices)\n",
    "        loader = torch.utils.data.DataLoader(\n",
    "            subset_data,\n",
    "            batch_size=batch_size,\n",
    "            num_workers=num_workers\n",
    "        )\n",
    "        teacher_loaders.append(loader)\n",
    "\n",
    "    return teacher_loaders\n",
    "\n",
    "teacher_loaders = get_data_loaders(train_data, num_teachers)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "2MwZOUVw_Q6A"
   },
   "source": [
    "Define a train student set of 9000 examples and 1000 test examples. Use 9000 samples of the dataset's test subset as unlabeled training points - they will be labeled using the teacher predictions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "bJjZ1DYX_Q6B"
   },
   "outputs": [],
   "source": [
    "student_train_data = Subset(test_data, list(range(9000)))\n",
    "student_test_data = Subset(test_data, list(range(9000, 10000)))\n",
    "\n",
    "student_train_loader = torch.utils.data.DataLoader(\n",
    "    student_train_data, batch_size=batch_size, \n",
    "    num_workers=num_workers\n",
    ")\n",
    "student_test_loader = torch.utils.data.DataLoader(\n",
    "    student_test_data, batch_size=batch_size, \n",
    "    num_workers=num_workers\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "K-0his1L_Q6D"
   },
   "source": [
    "## Defining models\n",
    "\n",
    "We are going to define a single model for all the teachers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Mg7fkeWD_Q6E"
   },
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "\n",
    "class Net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(1, 10, kernel_size=5)\n",
    "        self.conv2 = nn.Conv2d(10, 20, kernel_size=5)\n",
    "        self.conv2_drop = nn.Dropout2d()\n",
    "        self.fc1 = nn.Linear(320, 50)\n",
    "        self.fc2 = nn.Linear(50, 10)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = F.relu(F.max_pool2d(self.conv1(x), 2))\n",
    "        x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))\n",
    "        x = x.view(-1, 320)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.dropout(x, training=self.training)\n",
    "        x = self.fc2(x)\n",
    "        return F.log_softmax(x)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "w66owE9-_Q6H"
   },
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "def train(model, trainloader, criterion, optimizer, epochs=10, print_every=120):\n",
    "    model.to(device)\n",
    "    steps = 0\n",
    "    running_loss = 0\n",
    "    for e in range(epochs):\n",
    "        # Model in training mode, dropout is on\n",
    "        model.train()\n",
    "        for images, labels in trainloader:\n",
    "            images, labels = images.to(device), labels.to(device)\n",
    "            steps += 1\n",
    "            \n",
    "            optimizer.zero_grad()\n",
    "            \n",
    "            output = model.forward(images)\n",
    "            loss = criterion(output, labels)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            \n",
    "            running_loss += loss.item()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ItF8KZAv_Q6K"
   },
   "outputs": [],
   "source": [
    "def predict(model, dataloader):\n",
    "    outputs = torch.zeros(0, dtype=torch.long).to(device)\n",
    "    model.to(device)\n",
    "    model.eval()\n",
    "    for images, labels in dataloader:\n",
    "        images, labels = images.to(device), labels.to(device)\n",
    "        output = model.forward(images)\n",
    "        ps = torch.argmax(torch.exp(output), dim=1)\n",
    "        outputs = torch.cat((outputs, ps))\n",
    "    \n",
    "    return outputs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "SrACeiNU_Q6N"
   },
   "source": [
    "## Training all the teacher models\n",
    "\n",
    "Here we define and train the teachers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 103,
     "referenced_widgets": [
      "a7019a3be194415abc43ebe967a8417e",
      "42ffe71cf63f47d2b8196325ff08edb0",
      "cf7916b493ae4e6bbf64f60286178ed3",
      "8d7e8ebfcd1f4c4da07c9db9d479967f",
      "6ef387f1f9e1424bab89cc16d901009c",
      "5673fc4b225047fab55ec52b45ef9d25",
      "a341e400d7f04d9eae23b738f2549e35",
      "41bfc86d40da4a7ca22f4a857da32d9c"
     ]
    },
    "colab_type": "code",
    "id": "cmiezzAL_Q6O",
    "outputId": "9ac64250-10ea-4b74-8d39-e6422462fe50"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a7019a3be194415abc43ebe967a8417e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:21: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm.notebook import trange\n",
    "\n",
    "# Instantiate and train the models for each teacher\n",
    "def train_models(num_teachers):\n",
    "    models = []\n",
    "    for t in trange(num_teachers):\n",
    "        model = Net()\n",
    "        criterion = nn.NLLLoss()\n",
    "        optimizer = optim.Adam(model.parameters(), lr=0.003)\n",
    "        train(model, teacher_loaders[t], criterion, optimizer)\n",
    "        models.append(model)\n",
    "    return models\n",
    "\n",
    "models = train_models(num_teachers) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "fWK73FIX_Q6R"
   },
   "source": [
    "## Aggregated teacher\n",
    "\n",
    "This function predict the labels from all the dataset in each of the teachers, then return all the predictions and the maximum votation after adding laplacian noise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "jyrxvbfc_Q6R"
   },
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "vi48rANh_Q6U"
   },
   "outputs": [],
   "source": [
    "# define standard deviation for noise\n",
    "standard_deviation = 5.0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "JNwBqVpO_Q6X"
   },
   "source": [
    "# Aggregated teacher\n",
    "\n",
    "This function makes the predictions in all the teachers, count the votes and add noise, then returns the votation and the argmax results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "W_2mvbuc_Q6Y"
   },
   "outputs": [],
   "source": [
    "def aggregated_teacher(models, data_loader, standard_deviation=1.0):\n",
    "    preds = torch.torch.zeros((len(models), 9000), dtype=torch.long)\n",
    "    print('Running teacher predictions...')\n",
    "    for i, model in enumerate(models):\n",
    "        results = predict(model, data_loader)\n",
    "        preds[i] = results\n",
    "    \n",
    "    print('Calculating aggregates...')\n",
    "    labels = np.zeros(preds.shape[1]).astype(int)\n",
    "    for i, image_preds in enumerate(np.transpose(preds)):\n",
    "        label_counts = np.bincount(image_preds, minlength=10).astype(float)\n",
    "        label_counts += np.random.normal(0, standard_deviation, len(label_counts))\n",
    "        labels[i] = np.argmax(label_counts)\n",
    "    \n",
    "    return preds.numpy(), np.array(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 88
    },
    "colab_type": "code",
    "id": "JtPsYvpJ_Q6b",
    "outputId": "7a6b8d41-bea0-4745-aa96-107c5df97982",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running teacher predictions...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:21: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating aggregates...\n"
     ]
    }
   ],
   "source": [
    "teacher_models = models\n",
    "preds, student_labels = aggregated_teacher(teacher_models, student_train_loader, standard_deviation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "DF_cMtyN_Q6i"
   },
   "source": [
    "# Training the student\n",
    "\n",
    "Now we will train the student with the aggregated teacher labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nOnkfKCF_Q6i"
   },
   "outputs": [],
   "source": [
    "def student_loader(student_train_loader, labels):\n",
    "    for i, (data, _) in enumerate(iter(student_train_loader)):\n",
    "        yield data, torch.from_numpy(labels[i*len(data):(i+1)*len(data)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "MWjR-iHn_Q6l",
    "outputId": "47e5115f-c2a3-40c3-f3ea-d0b6de7089f2"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:21: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1/10..  Training Loss: 0.367..  Test Loss: 1.426..  Test Accuracy: 0.604\n",
      "Epoch: 1/10..  Training Loss: 0.214..  Test Loss: 0.652..  Test Accuracy: 0.805\n",
      "Epoch: 1/10..  Training Loss: 0.145..  Test Loss: 0.487..  Test Accuracy: 0.850\n",
      "Epoch: 1/10..  Training Loss: 0.101..  Test Loss: 0.396..  Test Accuracy: 0.882\n",
      "Epoch: 1/10..  Training Loss: 0.086..  Test Loss: 0.330..  Test Accuracy: 0.905\n",
      "Epoch: 2/10..  Training Loss: 0.120..  Test Loss: 0.368..  Test Accuracy: 0.903\n",
      "Epoch: 2/10..  Training Loss: 0.092..  Test Loss: 0.275..  Test Accuracy: 0.915\n",
      "Epoch: 2/10..  Training Loss: 0.078..  Test Loss: 0.282..  Test Accuracy: 0.910\n",
      "Epoch: 2/10..  Training Loss: 0.071..  Test Loss: 0.255..  Test Accuracy: 0.926\n",
      "Epoch: 2/10..  Training Loss: 0.055..  Test Loss: 0.234..  Test Accuracy: 0.930\n",
      "Epoch: 2/10..  Training Loss: 0.051..  Test Loss: 0.255..  Test Accuracy: 0.931\n",
      "Epoch: 3/10..  Training Loss: 0.093..  Test Loss: 0.247..  Test Accuracy: 0.925\n",
      "Epoch: 3/10..  Training Loss: 0.063..  Test Loss: 0.231..  Test Accuracy: 0.934\n",
      "Epoch: 3/10..  Training Loss: 0.064..  Test Loss: 0.232..  Test Accuracy: 0.927\n",
      "Epoch: 3/10..  Training Loss: 0.047..  Test Loss: 0.215..  Test Accuracy: 0.939\n",
      "Epoch: 3/10..  Training Loss: 0.043..  Test Loss: 0.218..  Test Accuracy: 0.940\n",
      "Epoch: 4/10..  Training Loss: 0.067..  Test Loss: 0.198..  Test Accuracy: 0.938\n",
      "Epoch: 4/10..  Training Loss: 0.062..  Test Loss: 0.232..  Test Accuracy: 0.931\n",
      "Epoch: 4/10..  Training Loss: 0.057..  Test Loss: 0.208..  Test Accuracy: 0.934\n",
      "Epoch: 4/10..  Training Loss: 0.055..  Test Loss: 0.201..  Test Accuracy: 0.930\n",
      "Epoch: 4/10..  Training Loss: 0.038..  Test Loss: 0.186..  Test Accuracy: 0.945\n",
      "Epoch: 4/10..  Training Loss: 0.041..  Test Loss: 0.210..  Test Accuracy: 0.942\n",
      "Epoch: 5/10..  Training Loss: 0.093..  Test Loss: 0.240..  Test Accuracy: 0.939\n",
      "Epoch: 5/10..  Training Loss: 0.058..  Test Loss: 0.197..  Test Accuracy: 0.935\n",
      "Epoch: 5/10..  Training Loss: 0.049..  Test Loss: 0.198..  Test Accuracy: 0.941\n",
      "Epoch: 5/10..  Training Loss: 0.047..  Test Loss: 0.201..  Test Accuracy: 0.945\n",
      "Epoch: 5/10..  Training Loss: 0.033..  Test Loss: 0.205..  Test Accuracy: 0.943\n",
      "Epoch: 5/10..  Training Loss: 0.036..  Test Loss: 0.197..  Test Accuracy: 0.949\n",
      "Epoch: 6/10..  Training Loss: 0.070..  Test Loss: 0.191..  Test Accuracy: 0.944\n",
      "Epoch: 6/10..  Training Loss: 0.047..  Test Loss: 0.182..  Test Accuracy: 0.946\n",
      "Epoch: 6/10..  Training Loss: 0.047..  Test Loss: 0.177..  Test Accuracy: 0.950\n",
      "Epoch: 6/10..  Training Loss: 0.037..  Test Loss: 0.184..  Test Accuracy: 0.949\n",
      "Epoch: 6/10..  Training Loss: 0.033..  Test Loss: 0.199..  Test Accuracy: 0.947\n",
      "Epoch: 7/10..  Training Loss: 0.056..  Test Loss: 0.176..  Test Accuracy: 0.953\n",
      "Epoch: 7/10..  Training Loss: 0.052..  Test Loss: 0.188..  Test Accuracy: 0.945\n",
      "Epoch: 7/10..  Training Loss: 0.048..  Test Loss: 0.191..  Test Accuracy: 0.947\n",
      "Epoch: 7/10..  Training Loss: 0.051..  Test Loss: 0.188..  Test Accuracy: 0.946\n",
      "Epoch: 7/10..  Training Loss: 0.032..  Test Loss: 0.202..  Test Accuracy: 0.947\n",
      "Epoch: 7/10..  Training Loss: 0.030..  Test Loss: 0.181..  Test Accuracy: 0.954\n",
      "Epoch: 8/10..  Training Loss: 0.062..  Test Loss: 0.188..  Test Accuracy: 0.948\n",
      "Epoch: 8/10..  Training Loss: 0.047..  Test Loss: 0.180..  Test Accuracy: 0.947\n",
      "Epoch: 8/10..  Training Loss: 0.039..  Test Loss: 0.178..  Test Accuracy: 0.954\n",
      "Epoch: 8/10..  Training Loss: 0.039..  Test Loss: 0.173..  Test Accuracy: 0.956\n",
      "Epoch: 8/10..  Training Loss: 0.026..  Test Loss: 0.184..  Test Accuracy: 0.950\n",
      "Epoch: 8/10..  Training Loss: 0.030..  Test Loss: 0.200..  Test Accuracy: 0.953\n",
      "Epoch: 9/10..  Training Loss: 0.095..  Test Loss: 0.191..  Test Accuracy: 0.948\n",
      "Epoch: 9/10..  Training Loss: 0.043..  Test Loss: 0.178..  Test Accuracy: 0.947\n",
      "Epoch: 9/10..  Training Loss: 0.041..  Test Loss: 0.177..  Test Accuracy: 0.948\n",
      "Epoch: 9/10..  Training Loss: 0.037..  Test Loss: 0.163..  Test Accuracy: 0.947\n",
      "Epoch: 9/10..  Training Loss: 0.028..  Test Loss: 0.175..  Test Accuracy: 0.949\n",
      "Epoch: 10/10..  Training Loss: 0.057..  Test Loss: 0.165..  Test Accuracy: 0.949\n",
      "Epoch: 10/10..  Training Loss: 0.051..  Test Loss: 0.181..  Test Accuracy: 0.946\n",
      "Epoch: 10/10..  Training Loss: 0.039..  Test Loss: 0.176..  Test Accuracy: 0.954\n",
      "Epoch: 10/10..  Training Loss: 0.042..  Test Loss: 0.178..  Test Accuracy: 0.945\n",
      "Epoch: 10/10..  Training Loss: 0.030..  Test Loss: 0.172..  Test Accuracy: 0.950\n",
      "Epoch: 10/10..  Training Loss: 0.026..  Test Loss: 0.190..  Test Accuracy: 0.952\n"
     ]
    }
   ],
   "source": [
    "student_model = Net()\n",
    "criterion = nn.NLLLoss()\n",
    "optimizer = optim.Adam(student_model.parameters(), lr=0.001)\n",
    "epochs = 10\n",
    "student_model.to(device)\n",
    "steps = 0\n",
    "running_loss = 0\n",
    "for e in range(epochs):\n",
    "    student_model.train()\n",
    "    train_loader = student_loader(student_train_loader, student_labels)\n",
    "    for images, labels in train_loader:\n",
    "        images, labels = images.to(device), labels.to(device)\n",
    "        steps += 1\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        output = student_model.forward(images)\n",
    "        loss = criterion(output, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        running_loss += loss.item()\n",
    "        if steps % 50 == 0:\n",
    "            test_loss = 0\n",
    "            accuracy = 0\n",
    "            student_model.eval()\n",
    "            with torch.no_grad():\n",
    "                for images, labels in student_test_loader:\n",
    "                    images, labels = images.to(device), labels.to(device)\n",
    "                    log_ps = student_model(images)\n",
    "                    test_loss += criterion(log_ps, labels).item()\n",
    "                    \n",
    "                    ps = torch.exp(log_ps)\n",
    "                    top_p, top_class = ps.topk(1, dim=1)\n",
    "                    equals = top_class == labels.view(*top_class.shape)\n",
    "                    accuracy += torch.mean(equals.type(torch.FloatTensor))\n",
    "            student_model.train()\n",
    "            print(\"Epoch: {}/{}.. \".format(e+1, epochs),\n",
    "                  \"Training Loss: {:.3f}.. \".format(running_loss/len(student_train_loader)),\n",
    "                  \"Test Loss: {:.3f}.. \".format(test_loss/len(student_test_loader)),\n",
    "                  \"Test Accuracy: {:.3f}\".format(accuracy/len(student_test_loader)))\n",
    "            running_loss = 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "tnDXbz1dyy13"
   },
   "source": [
    "# Privacy Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "dyoxKjqni1q1"
   },
   "outputs": [],
   "source": [
    "In Papernot and others (2018), they detail how data-dependent differential privacy bounds can be computed to estimate the cost of training the student. \n",
    "\n",
    "They provide a script to do this analysis based on the vote counts and the used standard deviation of the noise."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Ku2DLGSMi2v3"
   },
   "outputs": [],
   "source": [
    "!git clone https://github.com/tensorflow/privacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "4Z6WQvctjj-2"
   },
   "outputs": [],
   "source": [
    "%cd privacy/research/pate_2018/ICLR2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 136
    },
    "colab_type": "code",
    "id": "E8-GzZIJ455H",
    "outputId": "df2a8e11-f666-4b02-e99a-71b34407320f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data\t\t\t\t    plots_for_slides.py\n",
      "download.py\t\t\t    rdp_bucketized.py\n",
      "generate_figures.sh\t\t    rdp_cumulative.py\n",
      "generate_table_data_independent.sh  README.md\n",
      "generate_table.sh\t\t    smooth_sensitivity_table.py\n",
      "plot_ls_q.py\t\t\t    utility_queries_answered.py\n",
      "plot_partition.py\n"
     ]
    }
   ],
   "source": [
    "!ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "Q-YB9oMg_vZ-",
    "outputId": "cf73b6aa-b340-487c-bc4d-4c395255ea66"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100, 9000)"
      ]
     },
     "execution_count": 42,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ln_Vk86q5baY"
   },
   "outputs": [],
   "source": [
    "# put together the counts matrix:\n",
    "clean_votes = []\n",
    "for image_preds in np.transpose(preds):\n",
    "    label_counts = np.bincount(image_preds, minlength=10).astype(float)\n",
    "    clean_votes.append(label_counts)\n",
    "\n",
    "clean_votes = np.array(label_counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Y05DIkjJ6HsT"
   },
   "outputs": [],
   "source": [
    "with open('clean_votes.npy', 'wb') as file_obj:\n",
    "  np.save(file_obj, clean_votes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "dD_OuAoMAgE3"
   },
   "outputs": [],
   "source": [
    "#with open('labels_for_dump.npy', 'wb') as file_obj:\n",
    "#  np.save(file_obj, preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "0u-48maABIzy",
    "outputId": "9db2c4cb-abf3-4f25-f7d1-66632fe3af2d"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5.0"
      ]
     },
     "execution_count": 57,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "standard_deviation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "s9WSMiey478R",
    "outputId": "d2771a69-daa6-4d17-aa50-aaec90df6217"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading raw votes from clean_votes.npy\n",
      "Shape of the votes matrix = (9000, 10)\n",
      "Process all 9000 input rows. (Use --queries flag to truncate.)\n",
      "queries = 1000, E[answered] = 1000.00, E[eps] = 8.862 (std = 0.83114) at order = 4.00 (contribution from delta = 3.838)\n",
      "queries = 2000, E[answered] = 2000.00, E[eps] = 14.608 (std = 0.95292) at order = 3.00 (contribution from delta = 5.756)\n",
      "queries = 3000, E[answered] = 3000.00, E[eps] = 18.292 (std = 0.95182) at order = 2.50 (contribution from delta = 7.675)\n",
      "queries = 4000, E[answered] = 4000.00, E[eps] = 22.520 (std = 1.12738) at order = 2.50 (contribution from delta = 7.675)\n",
      "queries = 5000, E[answered] = 5000.00, E[eps] = 27.469 (std = 1.30883) at order = 2.50 (contribution from delta = 7.675)\n",
      "queries = 6000, E[answered] = 6000.00, E[eps] = 29.652 (std = 1.12725) at order = 2.00 (contribution from delta = 11.513)\n",
      "queries = 7000, E[answered] = 7000.00, E[eps] = 30.871 (std = 1.16386) at order = 2.00 (contribution from delta = 11.513)\n",
      "queries = 8000, E[answered] = 8000.00, E[eps] = 32.932 (std = 1.22456) at order = 2.00 (contribution from delta = 11.513)\n",
      "queries = 9000, E[answered] = 9000.00, E[eps] = 34.226 (std = 1.26461) at order = 2.00 (contribution from delta = 11.513)\n",
      "Checking conditions C5 and C6 for all orders.\n",
      "Conditions C5-C6 hold for all orders.\n",
      "100: E[answered queries] = 100.0, RDP at 2.0 goes from 11.745 to 12.805 +/- 0.302 (ss = 0.1012, beta = 0.1500, sigma_ss = 2.988)\n",
      "200: E[answered queries] = 200.0, RDP at 2.0 goes from 11.868 to 13.077 +/- 0.451 (ss = 0.1846, beta = 0.1500, sigma_ss = 2.446)\n",
      "300: E[answered queries] = 300.0, RDP at 2.0 goes from 11.941 to 13.230 +/- 0.531 (ss = 0.2352, beta = 0.1500, sigma_ss = 2.256)\n",
      "400: E[answered queries] = 400.0, RDP at 2.0 goes from 12.420 to 13.926 +/- 0.747 (ss = 0.3931, beta = 0.1500, sigma_ss = 1.901)\n",
      "500: E[answered queries] = 500.0, RDP at 2.0 goes from 12.652 to 14.262 +/- 0.852 (ss = 0.479, beta = 0.1500, sigma_ss = 1.780)\n",
      "600: E[answered queries] = 600.0, RDP at 2.0 goes from 13.245 to 15.022 +/- 1.019 (ss = 0.6258, beta = 0.1500, sigma_ss = 1.628)\n",
      "700: E[answered queries] = 700.0, RDP at 2.0 goes from 13.523 to 15.456 +/- 1.175 (ss = 0.7752, beta = 0.1500, sigma_ss = 1.516)\n",
      "800: E[answered queries] = 800.0, RDP at 2.0 goes from 13.536 to 15.534 +/- 1.239 (ss = 0.8396, beta = 0.1500, sigma_ss = 1.476)\n",
      "900: E[answered queries] = 900.0, RDP at 2.0 goes from 13.823 to 15.901 +/- 1.320 (ss = 0.9228, beta = 0.1500, sigma_ss = 1.430)\n",
      "1000: E[answered queries] = 1000.0, RDP at 2.0 goes from 14.105 to 16.265 +/- 1.401 (ss = 1.01, beta = 0.1500, sigma_ss = 1.388)\n",
      "1100: E[answered queries] = 1100.0, RDP at 2.0 goes from 14.407 to 16.702 +/- 1.536 (ss = 1.159, beta = 0.1500, sigma_ss = 1.326)\n",
      "1200: E[answered queries] = 1200.0, RDP at 2.0 goes from 14.879 to 17.312 +/- 1.675 (ss = 1.319, beta = 0.1500, sigma_ss = 1.270)\n",
      "1300: E[answered queries] = 1300.0, RDP at 2.0 goes from 15.299 to 17.876 +/- 1.784 (ss = 1.443, beta = 0.1550, sigma_ss = 1.236)\n",
      "1400: E[answered queries] = 1400.0, RDP at 2.0 goes from 15.346 to 18.037 +/- 1.860 (ss = 1.528, beta = 0.1600, sigma_ss = 1.217)\n",
      "1500: E[answered queries] = 1500.0, RDP at 2.0 goes from 15.661 to 18.503 +/- 1.973 (ss = 1.661, beta = 0.1650, sigma_ss = 1.188)\n",
      "1600: E[answered queries] = 1600.0, RDP at 2.0 goes from 16.142 to 19.068 +/- 2.017 (ss = 1.708, beta = 0.1700, sigma_ss = 1.180)\n",
      "1700: E[answered queries] = 1700.0, RDP at 2.0 goes from 16.501 to 19.501 +/- 2.090 (ss = 1.802, beta = 0.1700, sigma_ss = 1.160)\n",
      "1800: E[answered queries] = 1800.0, RDP at 2.0 goes from 16.859 to 19.993 +/- 2.182 (ss = 1.913, beta = 0.1750, sigma_ss = 1.141)\n",
      "1900: E[answered queries] = 1900.0, RDP at 2.0 goes from 17.100 to 20.324 +/- 2.228 (ss = 1.964, beta = 0.1800, sigma_ss = 1.134)\n",
      "2000: E[answered queries] = 2000.0, RDP at 2.0 goes from 17.642 to 20.962 +/- 2.323 (ss = 2.092, beta = 0.1800, sigma_ss = 1.111)\n",
      "2100: E[answered queries] = 2100.0, RDP at 2.0 goes from 17.912 to 21.378 +/- 2.423 (ss = 2.216, beta = 0.1850, sigma_ss = 1.093)\n",
      "2200: E[answered queries] = 2200.0, RDP at 2.0 goes from 18.130 to 21.647 +/- 2.474 (ss = 2.286, beta = 0.1850, sigma_ss = 1.082)\n",
      "2300: E[answered queries] = 2300.0, RDP at 2.0 goes from 18.373 to 21.978 +/- 2.511 (ss = 2.327, beta = 0.1900, sigma_ss = 1.079)\n",
      "2400: E[answered queries] = 2400.0, RDP at 2.0 goes from 18.679 to 22.411 +/- 2.638 (ss = 2.506, beta = 0.1900, sigma_ss = 1.053)\n",
      "2500: E[answered queries] = 2500.0, RDP at 2.0 goes from 18.747 to 22.512 +/- 2.672 (ss = 2.554, beta = 0.1900, sigma_ss = 1.046)\n",
      "2600: E[answered queries] = 2600.0, RDP at 2.0 goes from 19.021 to 22.849 +/- 2.784 (ss = 2.73, beta = 0.1850, sigma_ss = 1.020)\n",
      "2700: E[answered queries] = 2700.0, RDP at 2.0 goes from 19.214 to 23.114 +/- 2.807 (ss = 2.75, beta = 0.1900, sigma_ss = 1.021)\n",
      "2800: E[answered queries] = 2800.0, RDP at 2.0 goes from 19.702 to 23.721 +/- 2.872 (ss = 2.832, beta = 0.1950, sigma_ss = 1.014)\n",
      "2900: E[answered queries] = 2900.0, RDP at 2.0 goes from 19.979 to 24.102 +/- 2.918 (ss = 2.885, beta = 0.2000, sigma_ss = 1.011)\n",
      "3000: E[answered queries] = 3000.0, RDP at 2.0 goes from 20.228 to 24.463 +/- 3.031 (ss = 3.054, beta = 0.2000, sigma_ss = 0.992)\n",
      "3100: E[answered queries] = 3100.0, RDP at 2.0 goes from 20.391 to 24.644 +/- 3.048 (ss = 3.081, beta = 0.2000, sigma_ss = 0.989)\n",
      "3200: E[answered queries] = 3200.0, RDP at 2.0 goes from 21.056 to 25.473 +/- 3.149 (ss = 3.22, beta = 0.2050, sigma_ss = 0.978)\n",
      "3300: E[answered queries] = 3300.0, RDP at 2.0 goes from 21.318 to 25.781 +/- 3.195 (ss = 3.289, beta = 0.2050, sigma_ss = 0.971)\n",
      "3400: E[answered queries] = 3400.0, RDP at 2.0 goes from 21.394 to 25.865 +/- 3.266 (ss = 3.417, beta = 0.2000, sigma_ss = 0.956)\n",
      "3500: E[answered queries] = 3500.0, RDP at 2.0 goes from 21.827 to 26.355 +/- 3.324 (ss = 3.508, beta = 0.2000, sigma_ss = 0.947)\n",
      "3600: E[answered queries] = 3600.0, RDP at 2.0 goes from 22.154 to 26.798 +/- 3.376 (ss = 3.574, beta = 0.2050, sigma_ss = 0.945)\n",
      "3700: E[answered queries] = 3700.0, RDP at 2.0 goes from 22.287 to 26.993 +/- 3.438 (ss = 3.673, beta = 0.2050, sigma_ss = 0.936)\n",
      "3800: E[answered queries] = 3800.0, RDP at 2.0 goes from 22.859 to 27.675 +/- 3.480 (ss = 3.722, beta = 0.2100, sigma_ss = 0.935)\n",
      "3900: E[answered queries] = 3900.0, RDP at 2.0 goes from 23.447 to 28.357 +/- 3.574 (ss = 3.872, beta = 0.2100, sigma_ss = 0.923)\n",
      "4000: E[answered queries] = 4000.0, RDP at 2.0 goes from 23.668 to 28.667 +/- 3.662 (ss = 4.017, beta = 0.2100, sigma_ss = 0.912)\n",
      "4100: E[answered queries] = 4100.0, RDP at 2.0 goes from 24.020 to 29.071 +/- 3.716 (ss = 4.105, beta = 0.2100, sigma_ss = 0.905)\n",
      "4200: E[answered queries] = 4200.0, RDP at 2.0 goes from 24.405 to 29.534 +/- 3.793 (ss = 4.234, beta = 0.2100, sigma_ss = 0.896)\n",
      "4300: E[answered queries] = 4300.0, RDP at 2.0 goes from 25.113 to 30.355 +/- 3.906 (ss = 4.425, beta = 0.2100, sigma_ss = 0.883)\n",
      "4400: E[answered queries] = 4400.0, RDP at 2.0 goes from 25.522 to 30.839 +/- 3.980 (ss = 4.552, beta = 0.2100, sigma_ss = 0.874)\n",
      "4500: E[answered queries] = 4500.0, RDP at 2.0 goes from 25.913 to 31.259 +/- 4.009 (ss = 4.601, beta = 0.2100, sigma_ss = 0.871)\n",
      "4600: E[answered queries] = 4600.0, RDP at 2.0 goes from 26.233 to 31.559 +/- 4.058 (ss = 4.71, beta = 0.2050, sigma_ss = 0.862)\n",
      "4700: E[answered queries] = 4700.0, RDP at 2.0 goes from 26.537 to 31.917 +/- 4.112 (ss = 4.804, beta = 0.2050, sigma_ss = 0.856)\n",
      "4800: E[answered queries] = 4800.0, RDP at 2.0 goes from 26.713 to 32.135 +/- 4.155 (ss = 4.879, beta = 0.2050, sigma_ss = 0.852)\n",
      "4900: E[answered queries] = 4900.0, RDP at 2.0 goes from 27.274 to 32.774 +/- 4.295 (ss = 5.153, beta = 0.2000, sigma_ss = 0.833)\n",
      "5000: E[answered queries] = 5000.0, RDP at 2.0 goes from 27.674 to 33.186 +/- 4.307 (ss = 5.175, beta = 0.2000, sigma_ss = 0.832)\n",
      "5100: E[answered queries] = 5100.0, RDP at 2.0 goes from 27.946 to 33.507 +/- 4.356 (ss = 5.264, beta = 0.2000, sigma_ss = 0.828)\n",
      "5200: E[answered queries] = 5200.0, RDP at 2.0 goes from 27.946 to 33.511 +/- 4.360 (ss = 5.271, beta = 0.2000, sigma_ss = 0.827)\n",
      "5300: E[answered queries] = 5300.0, RDP at 2.0 goes from 28.074 to 33.673 +/- 4.394 (ss = 5.332, beta = 0.2000, sigma_ss = 0.824)\n",
      "5400: E[answered queries] = 5400.0, RDP at 2.0 goes from 28.158 to 33.807 +/- 4.382 (ss = 5.285, beta = 0.2050, sigma_ss = 0.829)\n",
      "5500: E[answered queries] = 5500.0, RDP at 2.0 goes from 28.308 to 33.973 +/- 4.397 (ss = 5.311, beta = 0.2050, sigma_ss = 0.828)\n",
      "5600: E[answered queries] = 5600.0, RDP at 2.0 goes from 28.548 to 34.227 +/- 4.412 (ss = 5.338, beta = 0.2050, sigma_ss = 0.827)\n",
      "5700: E[answered queries] = 5700.0, RDP at 2.0 goes from 28.820 to 34.558 +/- 4.470 (ss = 5.444, beta = 0.2050, sigma_ss = 0.821)\n",
      "5800: E[answered queries] = 5800.0, RDP at 2.0 goes from 28.918 to 34.734 +/- 4.479 (ss = 5.433, beta = 0.2100, sigma_ss = 0.824)\n",
      "5900: E[answered queries] = 5900.0, RDP at 2.0 goes from 29.003 to 34.835 +/- 4.495 (ss = 5.463, beta = 0.2100, sigma_ss = 0.823)\n",
      "6000: E[answered queries] = 6000.0, RDP at 2.0 goes from 29.652 to 35.560 +/- 4.571 (ss = 5.602, beta = 0.2100, sigma_ss = 0.816)\n",
      "6100: E[answered queries] = 6100.0, RDP at 2.0 goes from 29.795 to 35.733 +/- 4.602 (ss = 5.658, beta = 0.2100, sigma_ss = 0.813)\n",
      "6200: E[answered queries] = 6200.0, RDP at 2.0 goes from 29.835 to 35.813 +/- 4.642 (ss = 5.732, beta = 0.2100, sigma_ss = 0.810)\n",
      "6300: E[answered queries] = 6300.0, RDP at 2.0 goes from 29.835 to 35.813 +/- 4.642 (ss = 5.732, beta = 0.2100, sigma_ss = 0.810)\n",
      "6400: E[answered queries] = 6400.0, RDP at 2.0 goes from 29.995 to 35.985 +/- 4.654 (ss = 5.754, beta = 0.2100, sigma_ss = 0.809)\n",
      "6500: E[answered queries] = 6500.0, RDP at 2.0 goes from 30.180 to 36.217 +/- 4.701 (ss = 5.842, beta = 0.2100, sigma_ss = 0.805)\n",
      "6600: E[answered queries] = 6600.0, RDP at 2.0 goes from 30.355 to 36.436 +/- 4.745 (ss = 5.925, beta = 0.2100, sigma_ss = 0.801)\n",
      "6700: E[answered queries] = 6700.0, RDP at 2.0 goes from 30.515 to 36.608 +/- 4.757 (ss = 5.947, beta = 0.2100, sigma_ss = 0.800)\n",
      "6800: E[answered queries] = 6800.0, RDP at 2.0 goes from 30.643 to 36.784 +/- 4.805 (ss = 6.037, beta = 0.2100, sigma_ss = 0.796)\n",
      "6900: E[answered queries] = 6900.0, RDP at 2.0 goes from 30.803 to 37.009 +/- 4.793 (ss = 5.984, beta = 0.2150, sigma_ss = 0.801)\n",
      "7000: E[answered queries] = 7000.0, RDP at 2.0 goes from 30.871 to 37.109 +/- 4.825 (ss = 6.044, beta = 0.2150, sigma_ss = 0.798)\n",
      "7100: E[answered queries] = 7100.0, RDP at 2.0 goes from 30.951 to 37.198 +/- 4.833 (ss = 6.06, beta = 0.2150, sigma_ss = 0.798)\n",
      "7200: E[answered queries] = 7200.0, RDP at 2.0 goes from 31.114 to 37.383 +/- 4.857 (ss = 6.104, beta = 0.2150, sigma_ss = 0.796)\n",
      "7300: E[answered queries] = 7300.0, RDP at 2.0 goes from 31.482 to 37.786 +/- 4.968 (ss = 6.346, beta = 0.2100, sigma_ss = 0.783)\n",
      "7400: E[answered queries] = 7400.0, RDP at 2.0 goes from 31.485 to 37.800 +/- 4.979 (ss = 6.367, beta = 0.2100, sigma_ss = 0.782)\n",
      "7500: E[answered queries] = 7500.0, RDP at 2.0 goes from 31.855 to 38.265 +/- 4.997 (ss = 6.371, beta = 0.2150, sigma_ss = 0.784)\n",
      "7600: E[answered queries] = 7600.0, RDP at 2.0 goes from 32.206 to 38.671 +/- 5.052 (ss = 6.476, beta = 0.2150, sigma_ss = 0.780)\n",
      "7700: E[answered queries] = 7700.0, RDP at 2.0 goes from 32.206 to 38.671 +/- 5.052 (ss = 6.476, beta = 0.2150, sigma_ss = 0.780)\n",
      "7800: E[answered queries] = 7800.0, RDP at 2.0 goes from 32.368 to 38.860 +/- 5.079 (ss = 6.528, beta = 0.2150, sigma_ss = 0.778)\n",
      "7900: E[answered queries] = 7900.0, RDP at 2.0 goes from 32.688 to 39.222 +/- 5.120 (ss = 6.608, beta = 0.2150, sigma_ss = 0.775)\n",
      "8000: E[answered queries] = 8000.0, RDP at 2.0 goes from 32.932 to 39.501 +/- 5.156 (ss = 6.678, beta = 0.2150, sigma_ss = 0.772)\n",
      "8100: E[answered queries] = 8100.0, RDP at 2.0 goes from 33.067 to 39.658 +/- 5.178 (ss = 6.72, beta = 0.2150, sigma_ss = 0.771)\n",
      "8200: E[answered queries] = 8200.0, RDP at 2.0 goes from 33.238 to 39.863 +/- 5.211 (ss = 6.785, beta = 0.2150, sigma_ss = 0.768)\n",
      "8300: E[answered queries] = 8300.0, RDP at 2.0 goes from 33.560 to 40.225 +/- 5.253 (ss = 6.866, beta = 0.2150, sigma_ss = 0.765)\n",
      "8400: E[answered queries] = 8400.0, RDP at 2.0 goes from 33.880 to 40.568 +/- 5.276 (ss = 6.911, beta = 0.2150, sigma_ss = 0.763)\n",
      "8500: E[answered queries] = 8500.0, RDP at 2.0 goes from 33.960 to 40.659 +/- 5.287 (ss = 6.933, beta = 0.2150, sigma_ss = 0.763)\n",
      "8600: E[answered queries] = 8600.0, RDP at 2.0 goes from 34.066 to 40.785 +/- 5.306 (ss = 6.971, beta = 0.2150, sigma_ss = 0.761)\n",
      "8700: E[answered queries] = 8700.0, RDP at 2.0 goes from 34.066 to 40.789 +/- 5.309 (ss = 6.977, beta = 0.2150, sigma_ss = 0.761)\n",
      "8800: E[answered queries] = 8800.0, RDP at 2.0 goes from 34.066 to 40.789 +/- 5.309 (ss = 6.977, beta = 0.2150, sigma_ss = 0.761)\n",
      "8900: E[answered queries] = 8900.0, RDP at 2.0 goes from 34.146 to 40.869 +/- 5.309 (ss = 6.977, beta = 0.2150, sigma_ss = 0.761)\n",
      "9000: E[answered queries] = 9000.0, RDP at 2.0 goes from 34.226 to 40.959 +/- 5.320 (ss = 6.998, beta = 0.2150, sigma_ss = 0.760)\n",
      "Optimal beta = 0.2150, E[SS_beta] = 6.998, sigma_ss = 0.76\n"
     ]
    }
   ],
   "source": [
    "!python smooth_sensitivity_table.py  --sigma2=5.0 --counts_file=clean_votes.npy --delta=1e-5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "EoCa4P73Eyda"
   },
   "source": [
    "Data Independent Epsilon: 34.226\n",
    "\n",
    "Data Dependent Epsilon: 6.998"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "PATE.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "01c73e5d7b3e4c7bae717f0ee51e1058": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "0b4fe1a43b2b462c8a83c76764ba9e18": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "0fadbe1391d741f5982f6ff9a88dfb37": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_7bac61dd31f64b56bfa1d2d76ef2e7e9",
      "placeholder": "​",
      "style": "IPY_MODEL_9d36447c5a564e4cb156efdc209fe012",
      "value": " 8192/? [00:00&lt;00:00, 21000.59it/s]"
     }
    },
    "1a9ee64ef77e4160bc0e59c01099be18": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "1b60c8ec78c745b3876a79c53684cf91": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_3c6e3eec49b54b369faa256c4440ffdf",
      "placeholder": "​",
      "style": "IPY_MODEL_0b4fe1a43b2b462c8a83c76764ba9e18",
      "value": " 9920512/? [00:19&lt;00:00, 1498789.01it/s]"
     }
    },
    "20ee51f905aa4576bd7b6df533cb430a": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2a27418390754d9397bd06b49363809a": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_5ed4cdf05f514f6195381c0c6a83df89",
      "placeholder": "​",
      "style": "IPY_MODEL_a73fa9db86344b59afe151470d01326f",
      "value": " 1654784/? [00:01&lt;00:00, 1495009.57it/s]"
     }
    },
    "2fc36c2d96ec4ea1b42ce74a40072db1": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "30d23308b628438c9424a68bf73f4468": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "3c6e3eec49b54b369faa256c4440ffdf": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "3d2ff1e00e844fc08f54f19cd2dff484": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_5a74a153098947f191dfa69ff9b1c2e2",
      "placeholder": "​",
      "style": "IPY_MODEL_b2513595352f433f9a8b7a8cbb30f07b",
      "value": " 32768/? [00:01&lt;00:00, 23805.25it/s]"
     }
    },
    "3f96f5ab3d6d47bcb7948e36233e4983": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "41bfc86d40da4a7ca22f4a857da32d9c": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "42ffe71cf63f47d2b8196325ff08edb0": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "4f8b5f7987c84a2a9ddb01f62876a024": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "5673fc4b225047fab55ec52b45ef9d25": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "596367d5d9a14f6c9a1fd9139dfaa442": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "5a74a153098947f191dfa69ff9b1c2e2": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "5ed4cdf05f514f6195381c0c6a83df89": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "5f1007296f3c4071ad9c1ee99643f596": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "6ef387f1f9e1424bab89cc16d901009c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "74e87bd28def4a0187379d04fb4b83ed": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "7bac61dd31f64b56bfa1d2d76ef2e7e9": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "88c34b495ef84d18868c89d86ba74781": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_b22a21cee9b74cf9bf88335f39f5eafc",
       "IPY_MODEL_1b60c8ec78c745b3876a79c53684cf91"
      ],
      "layout": "IPY_MODEL_30d23308b628438c9424a68bf73f4468"
     }
    },
    "8d7e8ebfcd1f4c4da07c9db9d479967f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_41bfc86d40da4a7ca22f4a857da32d9c",
      "placeholder": "​",
      "style": "IPY_MODEL_a341e400d7f04d9eae23b738f2549e35",
      "value": " 100/100 [02:12&lt;00:00,  1.33s/it]"
     }
    },
    "94fc494383184117aa3748673053bbe7": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_d79e8a56a09545a0b19cb50d1486d81c",
       "IPY_MODEL_0fadbe1391d741f5982f6ff9a88dfb37"
      ],
      "layout": "IPY_MODEL_20ee51f905aa4576bd7b6df533cb430a"
     }
    },
    "9d36447c5a564e4cb156efdc209fe012": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "9eb486ae196945a7b0213f36d8dc3cdf": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "a341e400d7f04d9eae23b738f2549e35": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "a7019a3be194415abc43ebe967a8417e": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_cf7916b493ae4e6bbf64f60286178ed3",
       "IPY_MODEL_8d7e8ebfcd1f4c4da07c9db9d479967f"
      ],
      "layout": "IPY_MODEL_42ffe71cf63f47d2b8196325ff08edb0"
     }
    },
    "a73fa9db86344b59afe151470d01326f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "aaf968ebb58d4b88810087c6222c13ba": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_f2f58cbdbe464dd6b4240425ed69a567",
       "IPY_MODEL_2a27418390754d9397bd06b49363809a"
      ],
      "layout": "IPY_MODEL_1a9ee64ef77e4160bc0e59c01099be18"
     }
    },
    "b22a21cee9b74cf9bf88335f39f5eafc": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "info",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_3f96f5ab3d6d47bcb7948e36233e4983",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_bd429a24c41f4f0aa03da63558677bce",
      "value": 1
     }
    },
    "b2513595352f433f9a8b7a8cbb30f07b": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "bd429a24c41f4f0aa03da63558677bce": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "cf7916b493ae4e6bbf64f60286178ed3": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_5673fc4b225047fab55ec52b45ef9d25",
      "max": 100,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_6ef387f1f9e1424bab89cc16d901009c",
      "value": 100
     }
    },
    "d79e8a56a09545a0b19cb50d1486d81c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_2fc36c2d96ec4ea1b42ce74a40072db1",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_74e87bd28def4a0187379d04fb4b83ed",
      "value": 1
     }
    },
    "e12027df7fdf4427976c5f89d35c78cf": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_e3c0820d97734cf9a987f0ddf54fabbc",
       "IPY_MODEL_3d2ff1e00e844fc08f54f19cd2dff484"
      ],
      "layout": "IPY_MODEL_01c73e5d7b3e4c7bae717f0ee51e1058"
     }
    },
    "e3c0820d97734cf9a987f0ddf54fabbc": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_5f1007296f3c4071ad9c1ee99643f596",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_9eb486ae196945a7b0213f36d8dc3cdf",
      "value": 1
     }
    },
    "f2f58cbdbe464dd6b4240425ed69a567": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_4f8b5f7987c84a2a9ddb01f62876a024",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_596367d5d9a14f6c9a1fd9139dfaa442",
      "value": 1
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
